{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "A100"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install optimum[exporters]\n",
        "!pip install optimum[onnxruntime-gpu]\n",
        "!apt-get install tensorrt\n",
        "!mkdir tmp"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P1jB07DWvw9c",
        "outputId": "2689d07e-d66f-4cae-e2ef-468697f958fd"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting optimum[exporters]\n",
            "  Downloading optimum-1.9.0-py3-none-any.whl (327 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m327.4/327.4 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting coloredlogs (from optimum[exporters])\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum[exporters]) (1.11.1)\n",
            "Collecting transformers[sentencepiece]>=4.26.0 (from optimum[exporters])\n",
            "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m86.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch>=1.9 in /usr/local/lib/python3.10/dist-packages (from optimum[exporters]) (2.0.1+cu118)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from optimum[exporters]) (23.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from optimum[exporters]) (1.22.4)\n",
            "Collecting huggingface-hub>=0.8.0 (from optimum[exporters])\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m26.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets (from optimum[exporters])\n",
            "  Downloading datasets-2.13.1-py3-none-any.whl (486 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m486.2/486.2 kB\u001b[0m \u001b[31m44.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting onnx (from optimum[exporters])\n",
            "  Downloading onnx-1.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (14.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.6/14.6 MB\u001b[0m \u001b[31m82.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting onnxruntime (from optimum[exporters])\n",
            "  Downloading onnxruntime-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m109.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting timm (from optimum[exporters])\n",
            "  Downloading timm-0.9.2-py3-none-any.whl (2.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m96.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (3.12.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (4.65.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (4.6.3)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.9->optimum[exporters]) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.9->optimum[exporters]) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.9->optimum[exporters]) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.9->optimum[exporters]) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.9->optimum[exporters]) (16.0.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.26.0->optimum[exporters]) (2022.10.31)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers[sentencepiece]>=4.26.0->optimum[exporters])\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m91.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers[sentencepiece]>=4.26.0->optimum[exporters])\n",
            "  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m83.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentencepiece!=0.1.92,>=0.1.91 (from transformers[sentencepiece]>=4.26.0->optimum[exporters])\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m76.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: protobuf<=3.20.3 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.26.0->optimum[exporters]) (3.20.3)\n",
            "Collecting humanfriendly>=9.1 (from coloredlogs->optimum[exporters])\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[exporters]) (9.0.0)\n",
            "Collecting dill<0.3.7,>=0.3.0 (from datasets->optimum[exporters])\n",
            "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[exporters]) (1.5.3)\n",
            "Collecting xxhash (from datasets->optimum[exporters])\n",
            "  Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m25.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets->optimum[exporters])\n",
            "  Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[exporters]) (3.8.4)\n",
            "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime->optimum[exporters]) (23.5.26)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum[exporters]) (1.3.0)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from timm->optimum[exporters]) (0.15.2+cu118)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (2.0.12)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (1.3.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.8.0->optimum[exporters]) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.8.0->optimum[exporters]) (2023.5.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.8.0->optimum[exporters]) (3.4)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.9->optimum[exporters]) (2.1.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum[exporters]) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum[exporters]) (2022.7.1)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->timm->optimum[exporters]) (8.4.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets->optimum[exporters]) (1.16.0)\n",
            "Installing collected packages: tokenizers, sentencepiece, safetensors, xxhash, onnx, humanfriendly, dill, multiprocess, huggingface-hub, coloredlogs, transformers, onnxruntime, datasets, timm, optimum\n",
            "Successfully installed coloredlogs-15.0.1 datasets-2.13.1 dill-0.3.6 huggingface-hub-0.15.1 humanfriendly-10.0 multiprocess-0.70.14 onnx-1.14.0 onnxruntime-1.15.1 optimum-1.9.0 safetensors-0.3.1 sentencepiece-0.1.99 timm-0.9.2 tokenizers-0.13.3 transformers-4.30.2 xxhash-3.2.0\n",
            "Requirement already satisfied: optimum[onnxruntime-gpu] in /usr/local/lib/python3.10/dist-packages (1.9.0)\n",
            "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from optimum[onnxruntime-gpu]) (15.0.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum[onnxruntime-gpu]) (1.11.1)\n",
            "Requirement already satisfied: transformers[sentencepiece]>=4.26.0 in /usr/local/lib/python3.10/dist-packages (from optimum[onnxruntime-gpu]) (4.30.2)\n",
            "Requirement already satisfied: torch>=1.9 in /usr/local/lib/python3.10/dist-packages (from optimum[onnxruntime-gpu]) (2.0.1+cu118)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from optimum[onnxruntime-gpu]) (23.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from optimum[onnxruntime-gpu]) (1.22.4)\n",
            "Requirement already satisfied: huggingface-hub>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from optimum[onnxruntime-gpu]) (0.15.1)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from optimum[onnxruntime-gpu]) (2.13.1)\n",
            "Requirement already satisfied: onnx in /usr/local/lib/python3.10/dist-packages (from optimum[onnxruntime-gpu]) (1.14.0)\n",
            "Collecting onnxruntime-gpu>=1.9.0 (from optimum[onnxruntime-gpu])\n",
            "  Downloading onnxruntime_gpu-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m11.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting evaluate (from optimum[onnxruntime-gpu])\n",
            "  Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.20.1 in /usr/local/lib/python3.10/dist-packages (from optimum[onnxruntime-gpu]) (3.20.3)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[onnxruntime-gpu]) (9.0.0)\n",
            "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[onnxruntime-gpu]) (0.3.6)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[onnxruntime-gpu]) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[onnxruntime-gpu]) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[onnxruntime-gpu]) (4.65.0)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[onnxruntime-gpu]) (3.2.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[onnxruntime-gpu]) (0.70.14)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[onnxruntime-gpu]) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[onnxruntime-gpu]) (3.8.4)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[onnxruntime-gpu]) (6.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[onnxruntime-gpu]) (3.12.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[onnxruntime-gpu]) (4.6.3)\n",
            "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime-gpu>=1.9.0->optimum[onnxruntime-gpu]) (23.5.26)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.9->optimum[onnxruntime-gpu]) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.9->optimum[onnxruntime-gpu]) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.9->optimum[onnxruntime-gpu]) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.9->optimum[onnxruntime-gpu]) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.9->optimum[onnxruntime-gpu]) (16.0.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.26.0->optimum[onnxruntime-gpu]) (2022.10.31)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.26.0->optimum[onnxruntime-gpu]) (0.13.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.26.0->optimum[onnxruntime-gpu]) (0.3.1)\n",
            "Requirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.26.0->optimum[onnxruntime-gpu]) (0.1.99)\n",
            "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->optimum[onnxruntime-gpu]) (10.0)\n",
            "Collecting responses<0.19 (from evaluate->optimum[onnxruntime-gpu])\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum[onnxruntime-gpu]) (1.3.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[onnxruntime-gpu]) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[onnxruntime-gpu]) (2.0.12)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[onnxruntime-gpu]) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[onnxruntime-gpu]) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[onnxruntime-gpu]) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[onnxruntime-gpu]) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[onnxruntime-gpu]) (1.3.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets->optimum[onnxruntime-gpu]) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets->optimum[onnxruntime-gpu]) (2023.5.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets->optimum[onnxruntime-gpu]) (3.4)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.9->optimum[onnxruntime-gpu]) (2.1.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum[onnxruntime-gpu]) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum[onnxruntime-gpu]) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets->optimum[onnxruntime-gpu]) (1.16.0)\n",
            "Installing collected packages: responses, onnxruntime-gpu, evaluate\n",
            "Successfully installed evaluate-0.4.0 onnxruntime-gpu-1.15.1 responses-0.18.0\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  cuda-cccl-12-1 cuda-cccl-12-2 cuda-cudart-12-1 cuda-cudart-12-2\n",
            "  cuda-cudart-dev-12-1 cuda-cudart-dev-12-2 cuda-driver-dev-12-1\n",
            "  cuda-driver-dev-12-2 cuda-nvcc-12-1 cuda-toolkit-12-1-config-common\n",
            "  cuda-toolkit-12-2-config-common cuda-toolkit-12-config-common libcublas-12-2\n",
            "  libcublas-dev-12-2 libnvinfer-bin libnvinfer-dev libnvinfer-dispatch-dev\n",
            "  libnvinfer-dispatch8 libnvinfer-headers-dev libnvinfer-headers-plugin-dev\n",
            "  libnvinfer-lean-dev libnvinfer-lean8 libnvinfer-plugin-dev\n",
            "  libnvinfer-plugin8 libnvinfer-samples libnvinfer-vc-plugin-dev\n",
            "  libnvinfer-vc-plugin8 libnvinfer8 libnvonnxparsers-dev libnvonnxparsers8\n",
            "  libnvparsers-dev libnvparsers8\n",
            "The following NEW packages will be installed:\n",
            "  cuda-cccl-12-1 cuda-cccl-12-2 cuda-cudart-12-1 cuda-cudart-12-2\n",
            "  cuda-cudart-dev-12-1 cuda-cudart-dev-12-2 cuda-driver-dev-12-1\n",
            "  cuda-driver-dev-12-2 cuda-nvcc-12-1 cuda-toolkit-12-1-config-common\n",
            "  cuda-toolkit-12-2-config-common cuda-toolkit-12-config-common libcublas-12-2\n",
            "  libcublas-dev-12-2 libnvinfer-bin libnvinfer-dev libnvinfer-dispatch-dev\n",
            "  libnvinfer-dispatch8 libnvinfer-headers-dev libnvinfer-headers-plugin-dev\n",
            "  libnvinfer-lean-dev libnvinfer-lean8 libnvinfer-plugin-dev\n",
            "  libnvinfer-plugin8 libnvinfer-samples libnvinfer-vc-plugin-dev\n",
            "  libnvinfer-vc-plugin8 libnvinfer8 libnvonnxparsers-dev libnvonnxparsers8\n",
            "  libnvparsers-dev libnvparsers8 tensorrt\n",
            "0 upgraded, 33 newly installed, 0 to remove and 15 not upgraded.\n",
            "Need to get 2,046 MB of archives.\n",
            "After this operation, 5,609 MB of additional disk space will be used.\n",
            "Get:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-cccl-12-1 12.1.109-1 [1,060 kB]\n",
            "Get:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-cccl-12-2 12.2.53-1 [1,147 kB]\n",
            "Get:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-toolkit-12-config-common 12.2.53-1 [16.4 kB]\n",
            "Get:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-toolkit-12-1-config-common 12.1.105-1 [16.3 kB]\n",
            "Get:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-cudart-12-1 12.1.105-1 [158 kB]\n",
            "Get:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-toolkit-12-2-config-common 12.2.53-1 [16.3 kB]\n",
            "Get:7 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-cudart-12-2 12.2.53-1 [161 kB]\n",
            "Get:8 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-driver-dev-12-1 12.1.105-1 [27.8 kB]\n",
            "Get:9 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-cudart-dev-12-1 12.1.105-1 [894 kB]\n",
            "Get:10 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-driver-dev-12-2 12.2.53-1 [28.1 kB]\n",
            "Get:11 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-cudart-dev-12-2 12.2.53-1 [908 kB]\n",
            "Get:12 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  cuda-nvcc-12-1 12.1.105-1 [44.7 MB]\n",
            "Get:13 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libcublas-12-2 12.2.1.16-1 [244 MB]\n",
            "Get:14 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libcublas-dev-12-2 12.2.1.16-1 [255 MB]\n",
            "Get:15 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer8 8.6.1.6-1+cuda12.0 [492 MB]\n",
            "Get:16 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-lean8 8.6.1.6-1+cuda12.0 [9,144 kB]\n",
            "Get:17 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-plugin8 8.6.1.6-1+cuda12.0 [11.7 MB]\n",
            "Get:18 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-vc-plugin8 8.6.1.6-1+cuda12.0 [237 kB]\n",
            "Get:19 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-dispatch8 8.6.1.6-1+cuda12.0 [212 kB]\n",
            "Get:20 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvparsers8 8.6.1.6-1+cuda12.0 [805 kB]\n",
            "Get:21 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvonnxparsers8 8.6.1.6-1+cuda12.0 [712 kB]\n",
            "Get:22 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-bin 8.6.1.6-1+cuda12.0 [352 kB]\n",
            "Get:23 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-headers-dev 8.6.1.6-1+cuda12.0 [101 kB]\n",
            "Get:24 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-dev 8.6.1.6-1+cuda12.0 [490 MB]\n",
            "Get:25 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-dispatch-dev 8.6.1.6-1+cuda12.0 [13.8 MB]\n",
            "Get:26 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-headers-plugin-dev 8.6.1.6-1+cuda12.0 [8,312 B]\n",
            "Get:27 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-lean-dev 8.6.1.6-1+cuda12.0 [18.3 MB]\n",
            "Get:28 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-plugin-dev 8.6.1.6-1+cuda12.0 [11.8 MB]\n",
            "Get:29 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-vc-plugin-dev 8.6.1.6-1+cuda12.0 [99.3 kB]\n",
            "Get:30 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvparsers-dev 8.6.1.6-1+cuda12.0 [1,797 kB]\n",
            "Get:31 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvonnxparsers-dev 8.6.1.6-1+cuda12.0 [430 kB]\n",
            "Get:32 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  libnvinfer-samples 8.6.1.6-1+cuda12.0 [446 MB]\n",
            "Get:33 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  tensorrt 8.6.1.6-1+cuda12.0 [2,946 B]\n",
            "Fetched 2,046 MB in 25s (82.7 MB/s)\n",
            "Extracting templates from packages: 100%\n",
            "Selecting previously unselected package cuda-cccl-12-1.\n",
            "(Reading database ... 123069 files and directories currently installed.)\n",
            "Preparing to unpack .../00-cuda-cccl-12-1_12.1.109-1_amd64.deb ...\n",
            "Unpacking cuda-cccl-12-1 (12.1.109-1) ...\n",
            "Selecting previously unselected package cuda-cccl-12-2.\n",
            "Preparing to unpack .../01-cuda-cccl-12-2_12.2.53-1_amd64.deb ...\n",
            "Unpacking cuda-cccl-12-2 (12.2.53-1) ...\n",
            "Selecting previously unselected package cuda-toolkit-12-config-common.\n",
            "Preparing to unpack .../02-cuda-toolkit-12-config-common_12.2.53-1_all.deb ...\n",
            "Unpacking cuda-toolkit-12-config-common (12.2.53-1) ...\n",
            "Selecting previously unselected package cuda-toolkit-12-1-config-common.\n",
            "Preparing to unpack .../03-cuda-toolkit-12-1-config-common_12.1.105-1_all.deb ...\n",
            "Unpacking cuda-toolkit-12-1-config-common (12.1.105-1) ...\n",
            "Selecting previously unselected package cuda-cudart-12-1.\n",
            "Preparing to unpack .../04-cuda-cudart-12-1_12.1.105-1_amd64.deb ...\n",
            "Unpacking cuda-cudart-12-1 (12.1.105-1) ...\n",
            "Selecting previously unselected package cuda-toolkit-12-2-config-common.\n",
            "Preparing to unpack .../05-cuda-toolkit-12-2-config-common_12.2.53-1_all.deb ...\n",
            "Unpacking cuda-toolkit-12-2-config-common (12.2.53-1) ...\n",
            "Selecting previously unselected package cuda-cudart-12-2.\n",
            "Preparing to unpack .../06-cuda-cudart-12-2_12.2.53-1_amd64.deb ...\n",
            "Unpacking cuda-cudart-12-2 (12.2.53-1) ...\n",
            "Selecting previously unselected package cuda-driver-dev-12-1.\n",
            "Preparing to unpack .../07-cuda-driver-dev-12-1_12.1.105-1_amd64.deb ...\n",
            "Unpacking cuda-driver-dev-12-1 (12.1.105-1) ...\n",
            "Selecting previously unselected package cuda-cudart-dev-12-1.\n",
            "Preparing to unpack .../08-cuda-cudart-dev-12-1_12.1.105-1_amd64.deb ...\n",
            "Unpacking cuda-cudart-dev-12-1 (12.1.105-1) ...\n",
            "Selecting previously unselected package cuda-driver-dev-12-2.\n",
            "Preparing to unpack .../09-cuda-driver-dev-12-2_12.2.53-1_amd64.deb ...\n",
            "Unpacking cuda-driver-dev-12-2 (12.2.53-1) ...\n",
            "Selecting previously unselected package cuda-cudart-dev-12-2.\n",
            "Preparing to unpack .../10-cuda-cudart-dev-12-2_12.2.53-1_amd64.deb ...\n",
            "Unpacking cuda-cudart-dev-12-2 (12.2.53-1) ...\n",
            "Selecting previously unselected package cuda-nvcc-12-1.\n",
            "Preparing to unpack .../11-cuda-nvcc-12-1_12.1.105-1_amd64.deb ...\n",
            "Unpacking cuda-nvcc-12-1 (12.1.105-1) ...\n",
            "Selecting previously unselected package libcublas-12-2.\n",
            "Preparing to unpack .../12-libcublas-12-2_12.2.1.16-1_amd64.deb ...\n",
            "Unpacking libcublas-12-2 (12.2.1.16-1) ...\n",
            "Selecting previously unselected package libcublas-dev-12-2.\n",
            "Preparing to unpack .../13-libcublas-dev-12-2_12.2.1.16-1_amd64.deb ...\n",
            "Unpacking libcublas-dev-12-2 (12.2.1.16-1) ...\n",
            "Selecting previously unselected package libnvinfer8.\n",
            "Preparing to unpack .../14-libnvinfer8_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer8 (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-lean8.\n",
            "Preparing to unpack .../15-libnvinfer-lean8_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-lean8 (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-plugin8.\n",
            "Preparing to unpack .../16-libnvinfer-plugin8_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-plugin8 (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-vc-plugin8.\n",
            "Preparing to unpack .../17-libnvinfer-vc-plugin8_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-vc-plugin8 (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-dispatch8.\n",
            "Preparing to unpack .../18-libnvinfer-dispatch8_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-dispatch8 (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvparsers8.\n",
            "Preparing to unpack .../19-libnvparsers8_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvparsers8 (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvonnxparsers8.\n",
            "Preparing to unpack .../20-libnvonnxparsers8_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvonnxparsers8 (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-bin.\n",
            "Preparing to unpack .../21-libnvinfer-bin_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-bin (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-headers-dev.\n",
            "Preparing to unpack .../22-libnvinfer-headers-dev_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-headers-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-dev.\n",
            "Preparing to unpack .../23-libnvinfer-dev_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-dispatch-dev.\n",
            "Preparing to unpack .../24-libnvinfer-dispatch-dev_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-dispatch-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-headers-plugin-dev.\n",
            "Preparing to unpack .../25-libnvinfer-headers-plugin-dev_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-headers-plugin-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-lean-dev.\n",
            "Preparing to unpack .../26-libnvinfer-lean-dev_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-lean-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-plugin-dev.\n",
            "Preparing to unpack .../27-libnvinfer-plugin-dev_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-plugin-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-vc-plugin-dev.\n",
            "Preparing to unpack .../28-libnvinfer-vc-plugin-dev_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvinfer-vc-plugin-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvparsers-dev.\n",
            "Preparing to unpack .../29-libnvparsers-dev_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvparsers-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvonnxparsers-dev.\n",
            "Preparing to unpack .../30-libnvonnxparsers-dev_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking libnvonnxparsers-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package libnvinfer-samples.\n",
            "Preparing to unpack .../31-libnvinfer-samples_8.6.1.6-1+cuda12.0_all.deb ...\n",
            "Unpacking libnvinfer-samples (8.6.1.6-1+cuda12.0) ...\n",
            "Selecting previously unselected package tensorrt.\n",
            "Preparing to unpack .../32-tensorrt_8.6.1.6-1+cuda12.0_amd64.deb ...\n",
            "Unpacking tensorrt (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up cuda-driver-dev-12-1 (12.1.105-1) ...\n",
            "Setting up cuda-toolkit-12-2-config-common (12.2.53-1) ...\n",
            "Setting alternatives\n",
            "update-alternatives: using /usr/local/cuda-12.2 to provide /usr/local/cuda (cuda) in auto mode\n",
            "update-alternatives: using /usr/local/cuda-12.2 to provide /usr/local/cuda-12 (cuda-12) in auto mode\n",
            "Setting up libnvinfer-headers-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up cuda-cccl-12-2 (12.2.53-1) ...\n",
            "Setting up cuda-cccl-12-1 (12.1.109-1) ...\n",
            "Setting up libnvinfer-lean8 (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up cuda-toolkit-12-1-config-common (12.1.105-1) ...\n",
            "Setting alternatives\n",
            "Setting up libnvinfer-vc-plugin8 (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvinfer-dispatch8 (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvinfer-headers-plugin-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up cuda-driver-dev-12-2 (12.2.53-1) ...\n",
            "Setting up cuda-toolkit-12-config-common (12.2.53-1) ...\n",
            "Setting up libnvinfer-lean-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up cuda-cudart-12-2 (12.2.53-1) ...\n",
            "Setting up libnvinfer-vc-plugin-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up cuda-cudart-12-1 (12.1.105-1) ...\n",
            "Setting up libnvinfer-dispatch-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up cuda-cudart-dev-12-1 (12.1.105-1) ...\n",
            "Setting up cuda-nvcc-12-1 (12.1.105-1) ...\n",
            "Setting up cuda-cudart-dev-12-2 (12.2.53-1) ...\n",
            "Setting up libcublas-12-2 (12.2.1.16-1) ...\n",
            "Setting up libcublas-dev-12-2 (12.2.1.16-1) ...\n",
            "Setting up libnvinfer8 (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvparsers8 (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvinfer-plugin8 (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvonnxparsers8 (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvinfer-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvonnxparsers-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvparsers-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvinfer-plugin-dev (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvinfer-bin (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up libnvinfer-samples (8.6.1.6-1+cuda12.0) ...\n",
            "Setting up tensorrt (8.6.1.6-1+cuda12.0) ...\n",
            "Processing triggers for libc-bin (2.31-0ubuntu9.9) ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SCPK3aHVvtw4",
        "outputId": "74391802-047b-427e-d2eb-f1d14330c616"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Framework not specified. Using pt to export to ONNX.\n",
            "Some weights of the model checkpoint at silk-road/luotuo-bert were not used when initializing BertModel: ['mlp.fc.bias', 'mlp.fc.weight', 'mlp.dense.weight', 'mlp.dense.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertModel were not initialized from the model checkpoint at silk-road/luotuo-bert and are newly initialized: ['bert.pooler.dense.weight', 'bert.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "Using framework PyTorch: 2.0.1+cu118\n",
            "Overriding 1 configuration item(s)\n",
            "\t- use_cache -> False\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "============= Diagnostic Run torch.onnx.export version 2.0.1+cu118 =============\n",
            "verbose: False, log level: Level.ERROR\n",
            "======================= 0 NONE 0 NOTE 0 WARNING 0 ERROR ========================\n",
            "\n",
            "BaseModelOutput(last_hidden_state=tensor([[[-0.0357, -0.0044,  0.0364,  ..., -0.0586, -0.0512, -0.0331],\n",
            "         [-0.0491, -0.0073,  0.1938,  ..., -0.0451, -0.3034,  0.1354],\n",
            "         [-0.0247, -0.2675, -0.1023,  ..., -0.0344, -0.3823,  0.1558],\n",
            "         ...,\n",
            "         [ 0.0284,  0.0383, -0.1177,  ..., -0.1113, -0.1828,  0.1152],\n",
            "         [ 0.0918, -0.1546,  0.1217,  ..., -0.1437, -0.2900, -0.0599],\n",
            "         [-0.0357, -0.0044,  0.0364,  ..., -0.0586, -0.0512, -0.0331]]],\n",
            "       device='cuda:0'), hidden_states=None, attentions=None)\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModel\n",
        "from optimum.onnxruntime import ORTModelForQuestionAnswering, ORTModelForFeatureExtraction\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "# tokenizer = AutoTokenizer.from_pretrained(\"silk-road/luotuo-bert\")\n",
        "# model = AutoModel.from_pretrained(\"silk-road/luotuo-bert\", trust_remote_code=True).to(device)\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"silk-road/luotuo-bert\")\n",
        "\n",
        "# model = ORTModelForFeatureExtraction.from_pretrained(\"luotuo-bert-onnx\")\n",
        "\n",
        "provider_options = {\n",
        "\n",
        "    \"trt_engine_cache_enable\": True,\n",
        "\n",
        "    \"trt_engine_cache_path\": \"tmp/trt_cache_luotuo_example\"\n",
        "\n",
        "}\n",
        "\n",
        "ort_model = ORTModelForFeatureExtraction.from_pretrained(\"silk-road/luotuo-bert\",\n",
        "                                                         export=True,\n",
        "                                                         provider=\"TensorrtExecutionProvider\",\n",
        "                                                         provider_options=provider_options,\n",
        "                                                         )\n",
        "\n",
        "\n",
        "inputs = tokenizer(\"我是谁?\", \"Using luotuo with ONNX Runtime!\", return_tensors=\"pt\")\n",
        "\n",
        "outputs = ort_model(**inputs)\n",
        "\n",
        "print(outputs)\n",
        "\n",
        "assert ort_model.providers == [\"TensorrtExecutionProvider\", \"CUDAExecutionProvider\", \"CPUExecutionProvider\"]\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "mB-knQWOwDne"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}